{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: Variables and autograd\n",
    "-------------------------------\n",
    "\n",
    "A fully-connected ReLU network with one hidden layer and no biases, trained to\n",
    "predict y from x by minimizing squared Euclidean distance.\n",
    "\n",
    "This implementation computes the forward pass using operations on PyTorch\n",
    "Variables, and uses PyTorch autograd to compute gradients.\n",
    "\n",
    "A PyTorch Variable is a wrapper around a PyTorch Tensor, and represents a node\n",
    "in a computational graph. If x is a Variable then x.data is a Tensor giving its\n",
    "value, and x.grad is another Variable holding the gradient of x with respect to\n",
    "some scalar value.\n",
    "\n",
    "PyTorch Variables have the same API as PyTorch tensors: (almost) any operation\n",
    "you can do on a Tensor you can also do on a Variable; the difference is that\n",
    "autograd allows you to automatically compute gradients.\n",
    "\n",
    "Source Link: http://pytorch.org/tutorials/beginner/examples_autograd/two_layer_net_autograd.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Orinal Tutorial code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 32435778.0\n",
      "1 27566394.0\n",
      "2 26051080.0\n",
      "3 24010326.0\n",
      "4 19955918.0\n",
      "5 14555348.0\n",
      "6 9478826.0\n",
      "7 5788778.0\n",
      "8 3516427.25\n",
      "9 2225696.0\n",
      "10 1506760.0\n",
      "11 1093481.0\n",
      "12 841975.0\n",
      "13 676903.625\n",
      "14 560574.75\n",
      "15 473537.3125\n",
      "16 405448.90625\n",
      "17 350482.625\n",
      "18 305083.21875\n",
      "19 267155.21875\n",
      "20 235035.71875\n",
      "21 207574.125\n",
      "22 183923.4375\n",
      "23 163435.53125\n",
      "24 145631.25\n",
      "25 130102.625\n",
      "26 116509.1484375\n",
      "27 104564.2890625\n",
      "28 94048.59375\n",
      "29 84753.71875\n",
      "30 76521.421875\n",
      "31 69214.7109375\n",
      "32 62706.046875\n",
      "33 56897.3046875\n",
      "34 51704.33984375\n",
      "35 47069.859375\n",
      "36 42908.55859375\n",
      "37 39165.109375\n",
      "38 35795.76171875\n",
      "39 32754.109375\n",
      "40 30006.876953125\n",
      "41 27518.8515625\n",
      "42 25264.115234375\n",
      "43 23216.369140625\n",
      "44 21353.666015625\n",
      "45 19658.45703125\n",
      "46 18113.607421875\n",
      "47 16704.01953125\n",
      "48 15416.408203125\n",
      "49 14238.5224609375\n",
      "50 13160.0625\n",
      "51 12171.9287109375\n",
      "52 11265.3125\n",
      "53 10432.8701171875\n",
      "54 9668.044921875\n",
      "55 8964.833984375\n",
      "56 8317.6181640625\n",
      "57 7721.30419921875\n",
      "58 7171.51171875\n",
      "59 6664.42919921875\n",
      "60 6196.30419921875\n",
      "61 5764.0927734375\n",
      "62 5364.5947265625\n",
      "63 4995.0400390625\n",
      "64 4653.1015625\n",
      "65 4336.38720703125\n",
      "66 4043.021240234375\n",
      "67 3771.077880859375\n",
      "68 3518.874755859375\n",
      "69 3284.817626953125\n",
      "70 3067.6259765625\n",
      "71 2865.789306640625\n",
      "72 2678.25244140625\n",
      "73 2503.89453125\n",
      "74 2341.72509765625\n",
      "75 2190.846435546875\n",
      "76 2050.39208984375\n",
      "77 1919.6690673828125\n",
      "78 1797.885009765625\n",
      "79 1684.3411865234375\n",
      "80 1578.4573974609375\n",
      "81 1479.67333984375\n",
      "82 1387.479736328125\n",
      "83 1301.410888671875\n",
      "84 1221.0762939453125\n",
      "85 1146.031494140625\n",
      "86 1075.8671875\n",
      "87 1010.272216796875\n",
      "88 948.9007568359375\n",
      "89 891.5098266601562\n",
      "90 837.7998046875\n",
      "91 787.5068969726562\n",
      "92 740.4194946289062\n",
      "93 696.3106079101562\n",
      "94 655.0028076171875\n",
      "95 616.2792358398438\n",
      "96 579.982177734375\n",
      "97 545.9352416992188\n",
      "98 514.0079345703125\n",
      "99 484.0511779785156\n",
      "100 455.9252014160156\n",
      "101 429.5201721191406\n",
      "102 404.7314453125\n",
      "103 381.4388427734375\n",
      "104 359.5576477050781\n",
      "105 338.99920654296875\n",
      "106 319.6728210449219\n",
      "107 301.5019226074219\n",
      "108 284.4130554199219\n",
      "109 268.3400573730469\n",
      "110 253.22276306152344\n",
      "111 238.9974822998047\n",
      "112 225.6061553955078\n",
      "113 213.00425720214844\n",
      "114 201.13433837890625\n",
      "115 189.95681762695312\n",
      "116 179.42689514160156\n",
      "117 169.5087432861328\n",
      "118 160.16000366210938\n",
      "119 151.35186767578125\n",
      "120 143.04598999023438\n",
      "121 135.21571350097656\n",
      "122 127.83134460449219\n",
      "123 120.86808776855469\n",
      "124 114.2987060546875\n",
      "125 108.10271453857422\n",
      "126 102.25538635253906\n",
      "127 96.73597717285156\n",
      "128 91.52713012695312\n",
      "129 86.60786437988281\n",
      "130 81.9635009765625\n",
      "131 77.57568359375\n",
      "132 73.43417358398438\n",
      "133 69.5193862915039\n",
      "134 65.82295227050781\n",
      "135 62.32745361328125\n",
      "136 59.025634765625\n",
      "137 55.90573501586914\n",
      "138 52.9556999206543\n",
      "139 50.165836334228516\n",
      "140 47.52710723876953\n",
      "141 45.032527923583984\n",
      "142 42.67301940917969\n",
      "143 40.440547943115234\n",
      "144 38.32932662963867\n",
      "145 36.331077575683594\n",
      "146 34.440277099609375\n",
      "147 32.65059280395508\n",
      "148 30.957677841186523\n",
      "149 29.35322380065918\n",
      "150 27.835742950439453\n",
      "151 26.398983001708984\n",
      "152 25.037654876708984\n",
      "153 23.748390197753906\n",
      "154 22.52790641784668\n",
      "155 21.370847702026367\n",
      "156 20.276081085205078\n",
      "157 19.23801612854004\n",
      "158 18.255035400390625\n",
      "159 17.322978973388672\n",
      "160 16.440004348754883\n",
      "161 15.603426933288574\n",
      "162 14.810013771057129\n",
      "163 14.058195114135742\n",
      "164 13.34529972076416\n",
      "165 12.669474601745605\n",
      "166 12.02862548828125\n",
      "167 11.420948028564453\n",
      "168 10.84462833404541\n",
      "169 10.298273086547852\n",
      "170 9.779711723327637\n",
      "171 9.288039207458496\n",
      "172 8.821798324584961\n",
      "173 8.379277229309082\n",
      "174 7.959425926208496\n",
      "175 7.5608110427856445\n",
      "176 7.182663917541504\n",
      "177 6.824026107788086\n",
      "178 6.483461380004883\n",
      "179 6.160484790802002\n",
      "180 5.853707313537598\n",
      "181 5.562228202819824\n",
      "182 5.2857489585876465\n",
      "183 5.023428440093994\n",
      "184 4.774433135986328\n",
      "185 4.537771224975586\n",
      "186 4.313005447387695\n",
      "187 4.099523544311523\n",
      "188 3.897091865539551\n",
      "189 3.704416513442993\n",
      "190 3.521493673324585\n",
      "191 3.3480513095855713\n",
      "192 3.183245897293091\n",
      "193 3.026447296142578\n",
      "194 2.8776986598968506\n",
      "195 2.7362771034240723\n",
      "196 2.602015495300293\n",
      "197 2.4743895530700684\n",
      "198 2.35300874710083\n",
      "199 2.2376577854156494\n",
      "200 2.1283416748046875\n",
      "201 2.024136543273926\n",
      "202 1.9251426458358765\n",
      "203 1.8311978578567505\n",
      "204 1.7418278455734253\n",
      "205 1.6569589376449585\n",
      "206 1.576233983039856\n",
      "207 1.4994686841964722\n",
      "208 1.4265549182891846\n",
      "209 1.3570979833602905\n",
      "210 1.2911818027496338\n",
      "211 1.228397011756897\n",
      "212 1.1688439846038818\n",
      "213 1.1123112440109253\n",
      "214 1.0582867860794067\n",
      "215 1.0070831775665283\n",
      "216 0.9584788084030151\n",
      "217 0.9121385812759399\n",
      "218 0.8681320548057556\n",
      "219 0.8261667490005493\n",
      "220 0.7863146662712097\n",
      "221 0.7483559846878052\n",
      "222 0.7122479677200317\n",
      "223 0.677996814250946\n",
      "224 0.6454443335533142\n",
      "225 0.6143575310707092\n",
      "226 0.5848075747489929\n",
      "227 0.5567524433135986\n",
      "228 0.5299323797225952\n",
      "229 0.5044969320297241\n",
      "230 0.4802645146846771\n",
      "231 0.45727330446243286\n",
      "232 0.4353187084197998\n",
      "233 0.41444212198257446\n",
      "234 0.39465227723121643\n",
      "235 0.37575674057006836\n",
      "236 0.3577708303928375\n",
      "237 0.34066569805145264\n",
      "238 0.324382483959198\n",
      "239 0.30892589688301086\n",
      "240 0.2941271662712097\n",
      "241 0.2801261842250824\n",
      "242 0.26679858565330505\n",
      "243 0.25409072637557983\n",
      "244 0.24198678135871887\n",
      "245 0.2304302304983139\n",
      "246 0.21946148574352264\n",
      "247 0.20901453495025635\n",
      "248 0.19905783236026764\n",
      "249 0.18959571421146393\n",
      "250 0.18057577311992645\n",
      "251 0.17198176681995392\n",
      "252 0.16381782293319702\n",
      "253 0.15603198111057281\n",
      "254 0.1486414223909378\n",
      "255 0.1415988951921463\n",
      "256 0.13490118086338043\n",
      "257 0.12848640978336334\n",
      "258 0.1223902553319931\n",
      "259 0.11660647392272949\n",
      "260 0.11110442131757736\n",
      "261 0.10580969601869583\n",
      "262 0.10083150118589401\n",
      "263 0.09609221667051315\n",
      "264 0.09153228253126144\n",
      "265 0.08720364421606064\n",
      "266 0.0831153616309166\n",
      "267 0.07917824387550354\n",
      "268 0.07544625550508499\n",
      "269 0.0718822032213211\n",
      "270 0.06850933283567429\n",
      "271 0.06529312580823898\n",
      "272 0.06221548095345497\n",
      "273 0.05927516892552376\n",
      "274 0.056493040174245834\n",
      "275 0.05381253734230995\n",
      "276 0.0512966550886631\n",
      "277 0.04889260604977608\n",
      "278 0.0465969517827034\n",
      "279 0.044413045048713684\n",
      "280 0.042331889271736145\n",
      "281 0.040333379060029984\n",
      "282 0.038452133536338806\n",
      "283 0.036635540425777435\n",
      "284 0.034930672496557236\n",
      "285 0.0333024300634861\n",
      "286 0.031751055270433426\n",
      "287 0.030266741290688515\n",
      "288 0.028857210651040077\n",
      "289 0.027514392510056496\n",
      "290 0.026237059384584427\n",
      "291 0.025008494034409523\n",
      "292 0.02384433150291443\n",
      "293 0.022733770310878754\n",
      "294 0.021684542298316956\n",
      "295 0.020675279200077057\n",
      "296 0.01970774494111538\n",
      "297 0.018797406926751137\n",
      "298 0.017931915819644928\n",
      "299 0.017111316323280334\n",
      "300 0.01632092148065567\n",
      "301 0.015567156486213207\n",
      "302 0.014849958941340446\n",
      "303 0.014167353510856628\n",
      "304 0.013507981784641743\n",
      "305 0.012892743572592735\n",
      "306 0.012300229631364346\n",
      "307 0.011743215844035149\n",
      "308 0.011206476017832756\n",
      "309 0.010696016252040863\n",
      "310 0.010207979008555412\n",
      "311 0.009754852391779423\n",
      "312 0.009311926551163197\n",
      "313 0.008890357799828053\n",
      "314 0.008494292385876179\n",
      "315 0.008109879679977894\n",
      "316 0.00773945776745677\n",
      "317 0.0073946514166891575\n",
      "318 0.007064529694616795\n",
      "319 0.006745990365743637\n",
      "320 0.006450625602155924\n",
      "321 0.006165518891066313\n",
      "322 0.005896276328712702\n",
      "323 0.005637081805616617\n",
      "324 0.005382343661040068\n",
      "325 0.0051471637561917305\n",
      "326 0.004923258442431688\n",
      "327 0.004715548828244209\n",
      "328 0.004510185681283474\n",
      "329 0.004315443802624941\n",
      "330 0.0041280100122094154\n",
      "331 0.003954857587814331\n",
      "332 0.0037847936619073153\n",
      "333 0.0036223905626684427\n",
      "334 0.003468992654234171\n",
      "335 0.0033248525578528643\n",
      "336 0.0031862545292824507\n",
      "337 0.003050916362553835\n",
      "338 0.0029253631364554167\n",
      "339 0.0028064523357897997\n",
      "340 0.0026919269002974033\n",
      "341 0.0025820501614362\n",
      "342 0.0024784503038972616\n",
      "343 0.0023763244971632957\n",
      "344 0.0022835074923932552\n",
      "345 0.0021920909639447927\n",
      "346 0.002102951053529978\n",
      "347 0.002020670333877206\n",
      "348 0.0019420947646722198\n",
      "349 0.0018671798752620816\n",
      "350 0.001794661395251751\n",
      "351 0.001722082495689392\n",
      "352 0.0016582715325057507\n",
      "353 0.0015970293898135424\n",
      "354 0.0015355685027316213\n",
      "355 0.0014780131168663502\n",
      "356 0.0014233082765713334\n",
      "357 0.0013687493046745658\n",
      "358 0.0013180477544665337\n",
      "359 0.0012702604290097952\n",
      "360 0.001225689658895135\n",
      "361 0.0011814996832981706\n",
      "362 0.0011402420932427049\n",
      "363 0.001099239569157362\n",
      "364 0.0010615917854011059\n",
      "365 0.001025042962282896\n",
      "366 0.0009888361673802137\n",
      "367 0.000955046562012285\n",
      "368 0.0009221380460076034\n",
      "369 0.0008889632881619036\n",
      "370 0.0008593991515226662\n",
      "371 0.0008309672120958567\n",
      "372 0.0008037292282097042\n",
      "373 0.0007776526035740972\n",
      "374 0.0007521695806644857\n",
      "375 0.0007285813335329294\n",
      "376 0.0007034195004962385\n",
      "377 0.000681719568092376\n",
      "378 0.0006592939025722444\n",
      "379 0.0006393793155439198\n",
      "380 0.0006208677659742534\n",
      "381 0.0006000887951813638\n",
      "382 0.0005825297557748854\n",
      "383 0.0005665157805196941\n",
      "384 0.0005477395025081933\n",
      "385 0.0005314845475368202\n",
      "386 0.0005159893189556897\n",
      "387 0.0005009225569665432\n",
      "388 0.0004865403170697391\n",
      "389 0.0004730396904051304\n",
      "390 0.0004593569610733539\n",
      "391 0.00044523528777062893\n",
      "392 0.00043325836304575205\n",
      "393 0.0004210746265016496\n",
      "394 0.00040910570533014834\n",
      "395 0.00039748751441948116\n",
      "396 0.000387493782909587\n",
      "397 0.00037617210182361305\n",
      "398 0.00036647525848820806\n",
      "399 0.0003567223029676825\n",
      "400 0.00034797473927028477\n",
      "401 0.00033895333763211966\n",
      "402 0.00032968682353384793\n",
      "403 0.00032154397922568023\n",
      "404 0.0003131391713395715\n",
      "405 0.00030544662149623036\n",
      "406 0.00029714664560742676\n",
      "407 0.0002893109049182385\n",
      "408 0.0002819087530951947\n",
      "409 0.0002749561390373856\n",
      "410 0.00026832762523554265\n",
      "411 0.00026216518017463386\n",
      "412 0.00025594557519070804\n",
      "413 0.00024949354701675475\n",
      "414 0.00024369962920900434\n",
      "415 0.00023785245139151812\n",
      "416 0.00023264125047717243\n",
      "417 0.00022653571795672178\n",
      "418 0.00022201784304343164\n",
      "419 0.00021671711874660105\n",
      "420 0.00021150401153136045\n",
      "421 0.00020689761731773615\n",
      "422 0.00020252114336472005\n",
      "423 0.0001974786282517016\n",
      "424 0.00019323131709825248\n",
      "425 0.000188464269740507\n",
      "426 0.00018495054973755032\n",
      "427 0.00018141519103664905\n",
      "428 0.0001773326948750764\n",
      "429 0.00017402914818376303\n",
      "430 0.000169950450072065\n",
      "431 0.00016645916912239045\n",
      "432 0.0001628344034543261\n",
      "433 0.0001589115709066391\n",
      "434 0.00015632336726412177\n",
      "435 0.00015222531510517\n",
      "436 0.000149445011629723\n",
      "437 0.0001463216176489368\n",
      "438 0.00014396874757949263\n",
      "439 0.00014053055201657116\n",
      "440 0.00013784000475425273\n",
      "441 0.0001348070945823565\n",
      "442 0.00013259459228720516\n",
      "443 0.00013052164285909384\n",
      "444 0.00012782881094608456\n",
      "445 0.000124742480693385\n",
      "446 0.00012264159158803523\n",
      "447 0.00012016180698992684\n",
      "448 0.00011809932038886473\n",
      "449 0.00011597297998378053\n",
      "450 0.00011351861758157611\n",
      "451 0.00011169350182171911\n",
      "452 0.00010946241673082113\n",
      "453 0.00010773824760690331\n",
      "454 0.00010570598533377051\n",
      "455 0.00010400786413811147\n",
      "456 0.00010251440107822418\n",
      "457 0.00010060083150165156\n",
      "458 9.870569192571566e-05\n",
      "459 9.704846888780594e-05\n",
      "460 9.541302279103547e-05\n",
      "461 9.394421067554504e-05\n",
      "462 9.194108861265704e-05\n",
      "463 9.045435581356287e-05\n",
      "464 8.879650704329833e-05\n",
      "465 8.753318979870528e-05\n",
      "466 8.586893818574026e-05\n",
      "467 8.437341602984816e-05\n",
      "468 8.291036647278816e-05\n",
      "469 8.18604530650191e-05\n",
      "470 8.03914517746307e-05\n",
      "471 7.873805589042604e-05\n",
      "472 7.736693078186363e-05\n",
      "473 7.637366798007861e-05\n",
      "474 7.518636266468093e-05\n",
      "475 7.39914903533645e-05\n",
      "476 7.315529364859685e-05\n",
      "477 7.17911243555136e-05\n",
      "478 7.04867925378494e-05\n",
      "479 6.940733146620914e-05\n",
      "480 6.879060674691573e-05\n",
      "481 6.775077781639993e-05\n",
      "482 6.654511525994167e-05\n",
      "483 6.576895248144865e-05\n",
      "484 6.439434946514666e-05\n",
      "485 6.336852675303817e-05\n",
      "486 6.288062286330387e-05\n",
      "487 6.221487274160609e-05\n",
      "488 6.129639223217964e-05\n",
      "489 6.023428068147041e-05\n",
      "490 5.9260772104607895e-05\n",
      "491 5.820353180752136e-05\n",
      "492 5.7538167311577126e-05\n",
      "493 5.6756620324449614e-05\n",
      "494 5.620000229100697e-05\n",
      "495 5.5351345508825034e-05\n",
      "496 5.4518655815627426e-05\n",
      "497 5.384187898016535e-05\n",
      "498 5.3019775805296376e-05\n",
      "499 5.242873521638103e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold input and outputs, and wrap them in Variables.\n",
    "# Setting requires_grad=False indicates that we do not need to compute gradients\n",
    "# with respect to these Variables during the backward pass.\n",
    "x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)\n",
    "y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)\n",
    "\n",
    "# Create random Tensors for weights, and wrap them in Variables.\n",
    "# Setting requires_grad=True indicates that we want to compute gradients with\n",
    "# respect to these Variables during the backward pass.\n",
    "w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)\n",
    "w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y using operations on Variables; these\n",
    "    # are exactly the same operations we used to compute the forward pass using\n",
    "    # Tensors, but we do not need to keep references to intermediate values since\n",
    "    # we are not implementing the backward pass by hand.\n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "\n",
    "    # Compute and print loss using operations on Variables.\n",
    "    # Now loss is a Variable of shape (1,) and loss.data is a Tensor of shape\n",
    "    # (1,); loss.data[0] is a scalar value holding the loss.\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Use autograd to compute the backward pass. This call will compute the\n",
    "    # gradient of loss with respect to all Variables with requires_grad=True.\n",
    "    # After this call w1.grad and w2.grad will be Variables holding the gradient\n",
    "    # of the loss with respect to w1 and w2 respectively.\n",
    "    loss.backward()\n",
    "\n",
    "    # Update weights using gradient descent; w1.data and w2.data are Tensors,\n",
    "    # w1.grad and w2.grad are Variables and w1.grad.data and w2.grad.data are\n",
    "    # Tensors.\n",
    "    w1.data -= learning_rate * w1.grad.data\n",
    "    w2.data -= learning_rate * w2.grad.data\n",
    "\n",
    "    # Manually zero the gradients after updating weights\n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Without #annotation Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 40186456.0\n",
      "1 38679132.0\n",
      "2 38968388.0\n",
      "3 33943436.0\n",
      "4 23728504.0\n",
      "5 13225834.0\n",
      "6 6622705.0\n",
      "7 3394681.0\n",
      "8 1979982.25\n",
      "9 1330394.125\n",
      "10 992870.1875\n",
      "11 787713.1875\n",
      "12 646239.6875\n",
      "13 540151.8125\n",
      "14 456806.65625\n",
      "15 389530.5625\n",
      "16 334396.65625\n",
      "17 288725.4375\n",
      "18 250538.890625\n",
      "19 218422.75\n",
      "20 191237.9375\n",
      "21 168069.609375\n",
      "22 148211.390625\n",
      "23 131130.828125\n",
      "24 116368.078125\n",
      "25 103549.1015625\n",
      "26 92383.25\n",
      "27 82627.890625\n",
      "28 74086.9765625\n",
      "29 66576.0859375\n",
      "30 59958.71875\n",
      "31 54103.67578125\n",
      "32 48911.5234375\n",
      "33 44296.6875\n",
      "34 40186.65625\n",
      "35 36513.94921875\n",
      "36 33224.5390625\n",
      "37 30275.16796875\n",
      "38 27625.548828125\n",
      "39 25238.169921875\n",
      "40 23084.7109375\n",
      "41 21140.40234375\n",
      "42 19382.2421875\n",
      "43 17794.375\n",
      "44 16356.3828125\n",
      "45 15050.724609375\n",
      "46 13861.0244140625\n",
      "47 12775.8232421875\n",
      "48 11785.0283203125\n",
      "49 10879.19921875\n",
      "50 10050.177734375\n",
      "51 9290.921875\n",
      "52 8595.6328125\n",
      "53 7956.3818359375\n",
      "54 7369.29638671875\n",
      "55 6829.5087890625\n",
      "56 6332.99951171875\n",
      "57 5875.48974609375\n",
      "58 5453.9013671875\n",
      "59 5065.16455078125\n",
      "60 4706.1630859375\n",
      "61 4374.5986328125\n",
      "62 4068.26318359375\n",
      "63 3785.057373046875\n",
      "64 3522.9599609375\n",
      "65 3280.440673828125\n",
      "66 3055.732421875\n",
      "67 2847.498046875\n",
      "68 2654.4638671875\n",
      "69 2475.509521484375\n",
      "70 2309.460693359375\n",
      "71 2155.2041015625\n",
      "72 2012.138671875\n",
      "73 1879.2135009765625\n",
      "74 1755.7469482421875\n",
      "75 1640.8486328125\n",
      "76 1533.9361572265625\n",
      "77 1434.43115234375\n",
      "78 1341.8062744140625\n",
      "79 1255.4910888671875\n",
      "80 1175.0528564453125\n",
      "81 1100.07666015625\n",
      "82 1030.18212890625\n",
      "83 964.963134765625\n",
      "84 904.0951538085938\n",
      "85 847.2772216796875\n",
      "86 794.2327880859375\n",
      "87 744.6956787109375\n",
      "88 698.4111938476562\n",
      "89 655.1514892578125\n",
      "90 614.718994140625\n",
      "91 576.9138793945312\n",
      "92 541.549560546875\n",
      "93 508.47064208984375\n",
      "94 477.5164489746094\n",
      "95 448.5426330566406\n",
      "96 421.40673828125\n",
      "97 396.01519775390625\n",
      "98 372.222412109375\n",
      "99 349.9305419921875\n",
      "100 329.0315246582031\n",
      "101 309.4501953125\n",
      "102 291.08795166015625\n",
      "103 273.8704528808594\n",
      "104 257.716552734375\n",
      "105 242.5594482421875\n",
      "106 228.3358612060547\n",
      "107 214.9869384765625\n",
      "108 202.45895385742188\n",
      "109 190.6901092529297\n",
      "110 179.64283752441406\n",
      "111 169.26321411132812\n",
      "112 159.50820922851562\n",
      "113 150.34530639648438\n",
      "114 141.73123168945312\n",
      "115 133.63433837890625\n",
      "116 126.01654815673828\n",
      "117 118.8556900024414\n",
      "118 112.12059020996094\n",
      "119 105.78081512451172\n",
      "120 99.81742858886719\n",
      "121 94.20462036132812\n",
      "122 88.92206573486328\n",
      "123 83.9475326538086\n",
      "124 79.26388549804688\n",
      "125 74.85304260253906\n",
      "126 70.69768524169922\n",
      "127 66.78251647949219\n",
      "128 63.09407043457031\n",
      "129 59.61800003051758\n",
      "130 56.34225082397461\n",
      "131 53.253509521484375\n",
      "132 50.34139633178711\n",
      "133 47.594364166259766\n",
      "134 45.00265884399414\n",
      "135 42.5583610534668\n",
      "136 40.252933502197266\n",
      "137 38.077186584472656\n",
      "138 36.02317428588867\n",
      "139 34.08562088012695\n",
      "140 32.25478744506836\n",
      "141 30.52757453918457\n",
      "142 28.895307540893555\n",
      "143 27.355417251586914\n",
      "144 25.899614334106445\n",
      "145 24.523744583129883\n",
      "146 23.22480010986328\n",
      "147 21.99742889404297\n",
      "148 20.83702850341797\n",
      "149 19.739789962768555\n",
      "150 18.70262908935547\n",
      "151 17.721813201904297\n",
      "152 16.79459571838379\n",
      "153 15.91793441772461\n",
      "154 15.088418960571289\n",
      "155 14.30407428741455\n",
      "156 13.561330795288086\n",
      "157 12.85932445526123\n",
      "158 12.19398307800293\n",
      "159 11.564709663391113\n",
      "160 10.969618797302246\n",
      "161 10.405576705932617\n",
      "162 9.87154483795166\n",
      "163 9.366190910339355\n",
      "164 8.887100219726562\n",
      "165 8.43409538269043\n",
      "166 8.004973411560059\n",
      "167 7.597763538360596\n",
      "168 7.2118024826049805\n",
      "169 6.8467793464660645\n",
      "170 6.5004496574401855\n",
      "171 6.1725029945373535\n",
      "172 5.861181259155273\n",
      "173 5.56630277633667\n",
      "174 5.286921977996826\n",
      "175 5.021762847900391\n",
      "176 4.770262241363525\n",
      "177 4.531790256500244\n",
      "178 4.305636882781982\n",
      "179 4.091308116912842\n",
      "180 3.887568235397339\n",
      "181 3.6945676803588867\n",
      "182 3.5114388465881348\n",
      "183 3.3375930786132812\n",
      "184 3.1726107597351074\n",
      "185 3.016207695007324\n",
      "186 2.867400884628296\n",
      "187 2.7264297008514404\n",
      "188 2.592474937438965\n",
      "189 2.4650914669036865\n",
      "190 2.3444149494171143\n",
      "191 2.2297797203063965\n",
      "192 2.1209113597869873\n",
      "193 2.017430067062378\n",
      "194 1.9190629720687866\n",
      "195 1.8256983757019043\n",
      "196 1.7370508909225464\n",
      "197 1.6526890993118286\n",
      "198 1.572699785232544\n",
      "199 1.4966052770614624\n",
      "200 1.424222469329834\n",
      "201 1.355409860610962\n",
      "202 1.2901884317398071\n",
      "203 1.2280324697494507\n",
      "204 1.169055461883545\n",
      "205 1.1128530502319336\n",
      "206 1.05954110622406\n",
      "207 1.008791446685791\n",
      "208 0.9604357481002808\n",
      "209 0.9145569205284119\n",
      "210 0.8709543347358704\n",
      "211 0.8293684124946594\n",
      "212 0.7899613976478577\n",
      "213 0.752366840839386\n",
      "214 0.7166428565979004\n",
      "215 0.6825985312461853\n",
      "216 0.6503105163574219\n",
      "217 0.6194206476211548\n",
      "218 0.5901228189468384\n",
      "219 0.562242865562439\n",
      "220 0.5357013940811157\n",
      "221 0.5104950666427612\n",
      "222 0.48644062876701355\n",
      "223 0.4635425806045532\n",
      "224 0.4417398273944855\n",
      "225 0.4209977090358734\n",
      "226 0.40130388736724854\n",
      "227 0.38242873549461365\n",
      "228 0.36456814408302307\n",
      "229 0.347525417804718\n",
      "230 0.3312472999095917\n",
      "231 0.3158375322818756\n",
      "232 0.3010898232460022\n",
      "233 0.28697478771209717\n",
      "234 0.27366119623184204\n",
      "235 0.2609691023826599\n",
      "236 0.24886304140090942\n",
      "237 0.23731660842895508\n",
      "238 0.2262803614139557\n",
      "239 0.21583247184753418\n",
      "240 0.2058471441268921\n",
      "241 0.196340873837471\n",
      "242 0.1872681975364685\n",
      "243 0.17860576510429382\n",
      "244 0.17037777602672577\n",
      "245 0.1625455915927887\n",
      "246 0.15506352484226227\n",
      "247 0.14791612327098846\n",
      "248 0.1411421298980713\n",
      "249 0.13464044034481049\n",
      "250 0.12850286066532135\n",
      "251 0.12256930768489838\n",
      "252 0.11695105582475662\n",
      "253 0.11161136627197266\n",
      "254 0.10651013255119324\n",
      "255 0.1016458049416542\n",
      "256 0.09700240939855576\n",
      "257 0.09258302301168442\n",
      "258 0.08836983144283295\n",
      "259 0.08433201164007187\n",
      "260 0.08049328625202179\n",
      "261 0.0768105536699295\n",
      "262 0.07332019507884979\n",
      "263 0.06998355686664581\n",
      "264 0.06681248545646667\n",
      "265 0.06379641592502594\n",
      "266 0.060899388045072556\n",
      "267 0.05814617872238159\n",
      "268 0.05551472678780556\n",
      "269 0.05299372598528862\n",
      "270 0.05059764161705971\n",
      "271 0.048334211111068726\n",
      "272 0.04613755643367767\n",
      "273 0.044065024703741074\n",
      "274 0.042069803923368454\n",
      "275 0.040179066359996796\n",
      "276 0.03837977349758148\n",
      "277 0.03666093572974205\n",
      "278 0.03500114008784294\n",
      "279 0.03342592716217041\n",
      "280 0.0319410115480423\n",
      "281 0.030499020591378212\n",
      "282 0.02913903072476387\n",
      "283 0.02782309241592884\n",
      "284 0.026595456525683403\n",
      "285 0.02540198154747486\n",
      "286 0.024271950125694275\n",
      "287 0.023187385872006416\n",
      "288 0.02216559648513794\n",
      "289 0.021173251792788506\n",
      "290 0.020244017243385315\n",
      "291 0.019353995099663734\n",
      "292 0.018498018383979797\n",
      "293 0.01768265850841999\n",
      "294 0.01688966155052185\n",
      "295 0.016148032620549202\n",
      "296 0.015434575267136097\n",
      "297 0.014759383164346218\n",
      "298 0.014110985212028027\n",
      "299 0.013494225218892097\n",
      "300 0.01290376391261816\n",
      "301 0.012341517023742199\n",
      "302 0.011808829382061958\n",
      "303 0.011290243826806545\n",
      "304 0.010805011726915836\n",
      "305 0.010335385799407959\n",
      "306 0.009885091334581375\n",
      "307 0.009457249194383621\n",
      "308 0.009046237915754318\n",
      "309 0.008661866188049316\n",
      "310 0.008285137824714184\n",
      "311 0.007929414510726929\n",
      "312 0.007590869441628456\n",
      "313 0.007262866012752056\n",
      "314 0.0069563863798975945\n",
      "315 0.006663523148745298\n",
      "316 0.006384828127920628\n",
      "317 0.006112134549766779\n",
      "318 0.005851438269019127\n",
      "319 0.0056062741205096245\n",
      "320 0.00537175964564085\n",
      "321 0.005148782394826412\n",
      "322 0.004935136530548334\n",
      "323 0.004731562454253435\n",
      "324 0.0045364066027104855\n",
      "325 0.004345775116235018\n",
      "326 0.004173437599092722\n",
      "327 0.003999879118055105\n",
      "328 0.0038308408111333847\n",
      "329 0.003683453658595681\n",
      "330 0.0035310264211148024\n",
      "331 0.0033877366222441196\n",
      "332 0.003256581025198102\n",
      "333 0.0031223671976476908\n",
      "334 0.0029984277207404375\n",
      "335 0.0028822943568229675\n",
      "336 0.0027693244628608227\n",
      "337 0.00265858368948102\n",
      "338 0.0025563579984009266\n",
      "339 0.0024574329145252705\n",
      "340 0.0023644149769097567\n",
      "341 0.002272252691909671\n",
      "342 0.002185596153140068\n",
      "343 0.002102822531014681\n",
      "344 0.0020232468377798796\n",
      "345 0.0019478051690384746\n",
      "346 0.0018751168390735984\n",
      "347 0.001808185363188386\n",
      "348 0.001742549822665751\n",
      "349 0.0016792298993095756\n",
      "350 0.0016173975309357047\n",
      "351 0.0015584275824949145\n",
      "352 0.001502386061474681\n",
      "353 0.001448300899937749\n",
      "354 0.001398555119521916\n",
      "355 0.0013511525467038155\n",
      "356 0.0013028301764279604\n",
      "357 0.0012585241347551346\n",
      "358 0.0012145375367254019\n",
      "359 0.0011739114997908473\n",
      "360 0.0011331281857565045\n",
      "361 0.0010924260132014751\n",
      "362 0.0010557913919910789\n",
      "363 0.0010213990462943912\n",
      "364 0.0009879807475954294\n",
      "365 0.0009554650750942528\n",
      "366 0.0009229417773894966\n",
      "367 0.0008949420880526304\n",
      "368 0.0008663532207719982\n",
      "369 0.0008381580701097846\n",
      "370 0.0008113574585877359\n",
      "371 0.0007857006276026368\n",
      "372 0.0007617896189913154\n",
      "373 0.0007397559238597751\n",
      "374 0.0007151567842811346\n",
      "375 0.0006938710575923324\n",
      "376 0.0006734378985129297\n",
      "377 0.0006529996171593666\n",
      "378 0.0006335967336781323\n",
      "379 0.0006143978098407388\n",
      "380 0.0005968164186924696\n",
      "381 0.0005793722230009735\n",
      "382 0.0005626695929095149\n",
      "383 0.0005463548004627228\n",
      "384 0.0005311206332407892\n",
      "385 0.0005154609098099172\n",
      "386 0.0005020044627599418\n",
      "387 0.00048798046191222966\n",
      "388 0.0004734635294880718\n",
      "389 0.0004610535397659987\n",
      "390 0.000447492056991905\n",
      "391 0.0004355135024525225\n",
      "392 0.0004237384127918631\n",
      "393 0.00041163500281982124\n",
      "394 0.0004003714129794389\n",
      "395 0.00039030113839544356\n",
      "396 0.0003801930870395154\n",
      "397 0.0003705251729115844\n",
      "398 0.0003610638377722353\n",
      "399 0.00035132194170728326\n",
      "400 0.0003425439936108887\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "401 0.0003340636030770838\n",
      "402 0.00032506551360711455\n",
      "403 0.0003161711501888931\n",
      "404 0.0003092814004048705\n",
      "405 0.00030239493935368955\n",
      "406 0.0002952552167698741\n",
      "407 0.000287755043245852\n",
      "408 0.0002808802528306842\n",
      "409 0.0002740605268627405\n",
      "410 0.000267561903456226\n",
      "411 0.00026169815100729465\n",
      "412 0.0002552659425418824\n",
      "413 0.00025056549930013716\n",
      "414 0.00024478178238496184\n",
      "415 0.00023870989389251918\n",
      "416 0.00023343593056779355\n",
      "417 0.00022792293748352677\n",
      "418 0.0002231706603197381\n",
      "419 0.0002182705793529749\n",
      "420 0.00021322461543604732\n",
      "421 0.00020859550568275154\n",
      "422 0.00020418758504092693\n",
      "423 0.00020026958372909576\n",
      "424 0.00019639609672594815\n",
      "425 0.0001916886103572324\n",
      "426 0.00018780511163640767\n",
      "427 0.00018379111133981496\n",
      "428 0.00018046698824036866\n",
      "429 0.00017712355474941432\n",
      "430 0.00017338921315968037\n",
      "431 0.00017000782827381045\n",
      "432 0.00016632885672152042\n",
      "433 0.00016287063772324473\n",
      "434 0.0001598523376742378\n",
      "435 0.00015615053416695446\n",
      "436 0.00015361698751803488\n",
      "437 0.00015057843120303005\n",
      "438 0.00014783203369006515\n",
      "439 0.00014559918781742454\n",
      "440 0.00014271460531745106\n",
      "441 0.00014028509031049907\n",
      "442 0.00013740963186137378\n",
      "443 0.0001350482925772667\n",
      "444 0.000132446875795722\n",
      "445 0.0001301044540014118\n",
      "446 0.00012761782272718847\n",
      "447 0.00012544069613795727\n",
      "448 0.000123082849313505\n",
      "449 0.00012100388266844675\n",
      "450 0.00011869431182276458\n",
      "451 0.00011705602810252458\n",
      "452 0.00011496804654598236\n",
      "453 0.00011290060501778498\n",
      "454 0.00011094247020082548\n",
      "455 0.00010938636114588007\n",
      "456 0.0001072946542990394\n",
      "457 0.00010553791798884049\n",
      "458 0.00010359418229199946\n",
      "459 0.00010195993672823533\n",
      "460 9.984480129787698e-05\n",
      "461 9.84967773547396e-05\n",
      "462 9.65670624282211e-05\n",
      "463 9.553187555866316e-05\n",
      "464 9.388753824168816e-05\n",
      "465 9.255165787180886e-05\n",
      "466 9.117564331972972e-05\n",
      "467 8.951860218076035e-05\n",
      "468 8.845384581945837e-05\n",
      "469 8.705150685273111e-05\n",
      "470 8.574181265430525e-05\n",
      "471 8.421832899330184e-05\n",
      "472 8.274374704342335e-05\n",
      "473 8.187977800844237e-05\n",
      "474 8.050817268667743e-05\n",
      "475 7.928058039397001e-05\n",
      "476 7.837614975869656e-05\n",
      "477 7.71679769968614e-05\n",
      "478 7.6190015533939e-05\n",
      "479 7.534220640081912e-05\n",
      "480 7.422973430948332e-05\n",
      "481 7.308257045224309e-05\n",
      "482 7.197384547907859e-05\n",
      "483 7.104730320861563e-05\n",
      "484 7.009852561168373e-05\n",
      "485 6.922698230482638e-05\n",
      "486 6.838269473519176e-05\n",
      "487 6.736635987181216e-05\n",
      "488 6.669968570349738e-05\n",
      "489 6.58267381368205e-05\n",
      "490 6.4809326431714e-05\n",
      "491 6.400355778168887e-05\n",
      "492 6.314330676104873e-05\n",
      "493 6.247122655622661e-05\n",
      "494 6.173907604534179e-05\n",
      "495 6.082337858970277e-05\n",
      "496 6.0319522162899375e-05\n",
      "497 5.9619305829983205e-05\n",
      "498 5.8785797591554e-05\n",
      "499 5.788941416540183e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# dtype = torch.cuda.FloatTensor # Uncomment this to run on GPU\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold input and outputs\n",
    "x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)\n",
    "y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)\n",
    "\n",
    "# Create random Tensors for weights, and wrap them in Variables.\n",
    "w1 = Variable(torch.randn(D_in, H).type(dtype), requires_grad=True)\n",
    "w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # Forward pass\n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Use autograd to compute the backward pass\n",
    "    loss.backward()\n",
    "\n",
    "    # Update weights using gradient descent\n",
    "    w1.data -= learning_rate * w1.grad.data\n",
    "    w2.data -= learning_rate * w2.grad.data\n",
    "\n",
    "    # Manually zero the gradients after updating weights\n",
    "    w1.grad.data.zero_()\n",
    "    w2.grad.data.zero_()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
